{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-ConvNet Architectures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, we concatenate *multiple parallel convolutional nets together* to classify IMDB movie reviews by their sentiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import imdb\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Model # new!\n",
    "from keras.layers import Input, concatenate # new! \n",
    "from keras.layers import Dense, Dropout, Embedding, SpatialDropout1D, Conv1D, GlobalMaxPooling1D\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "import os\n",
    "from sklearn.metrics import roc_auc_score \n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# output directory name:\n",
    "output_dir = 'model_output/multiconv'\n",
    "\n",
    "# training:\n",
    "epochs = 4\n",
    "batch_size = 128\n",
    "\n",
    "# vector-space embedding: \n",
    "n_dim = 64\n",
    "n_unique_words = 5000 \n",
    "max_review_length = 400\n",
    "pad_type = trunc_type = 'pre'\n",
    "drop_embed = 0.2 \n",
    "\n",
    "# convolutional layer architecture:\n",
    "n_conv_1 = n_conv_2 = n_conv_3 = 256 \n",
    "k_conv_1 = 3\n",
    "k_conv_2 = 2\n",
    "k_conv_3 = 4\n",
    "\n",
    "# dense layer architecture: \n",
    "n_dense = 256\n",
    "dropout = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/text-datasets/imdb.npz\n",
      "17276928/17464789 [============================>.] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)\n",
    "x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Design neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# start with conv_1 only and no concat\n",
    "# add conv_2\n",
    "# add conv_3\n",
    "# add dense_2\n",
    "\n",
    "input_layer = Input(shape=(max_review_length,), dtype='int16', name='input') # supports integers +/- 32.7k \n",
    "embedding_layer = Embedding(n_unique_words, n_dim, input_length=max_review_length, name='embedding')(input_layer)\n",
    "drop_embed_layer = SpatialDropout1D(drop_embed, name='drop_embed')(embedding_layer)\n",
    "\n",
    "conv_1 = Conv1D(n_conv_1, k_conv_1, activation='relu', name='conv_1')(drop_embed_layer)\n",
    "maxp_1 = GlobalMaxPooling1D(name='maxp_1')(conv_1)\n",
    "\n",
    "conv_2 = Conv1D(n_conv_2, k_conv_2, activation='relu', name='conv_2')(drop_embed_layer)\n",
    "maxp_2 = GlobalMaxPooling1D(name='maxp_2')(conv_2)\n",
    "\n",
    "conv_3 = Conv1D(n_conv_3, k_conv_3, activation='relu', name='conv_3')(drop_embed_layer)\n",
    "maxp_3 = GlobalMaxPooling1D(name='maxp_3')(conv_3)\n",
    "\n",
    "concat = concatenate([maxp_1, maxp_2, maxp_3])\n",
    "\n",
    "dense_layer = Dense(n_dense, activation='relu', name='dense')(concat)\n",
    "drop_dense_layer = Dropout(dropout, name='drop_dense')(dense_layer)\n",
    "dense_2 = Dense(64, activation='relu', name='dense_2')(drop_dense_layer)\n",
    "dropout_2 = Dropout(dropout, name='drop_dense_2')(dense_2)\n",
    "\n",
    "predictions = Dense(1, activation='sigmoid', name='output')(dropout_2)\n",
    "\n",
    "model = Model(input_layer, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input (InputLayer)               (None, 400)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding (Embedding)            (None, 400, 64)       320000      input[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "drop_embed (SpatialDropout1D)    (None, 400, 64)       0           embedding[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "conv_1 (Conv1D)                  (None, 398, 256)      49408       drop_embed[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "conv_2 (Conv1D)                  (None, 399, 256)      33024       drop_embed[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "conv_3 (Conv1D)                  (None, 397, 256)      65792       drop_embed[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "maxp_1 (GlobalMaxPooling1D)      (None, 256)           0           conv_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "maxp_2 (GlobalMaxPooling1D)      (None, 256)           0           conv_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "maxp_3 (GlobalMaxPooling1D)      (None, 256)           0           conv_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 768)           0           maxp_1[0][0]                     \n",
      "                                                                   maxp_2[0][0]                     \n",
      "                                                                   maxp_3[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense (Dense)                    (None, 256)           196864      concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "drop_dense (Dropout)             (None, 256)           0           dense[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 64)            16448       drop_dense[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "drop_dense_2 (Dropout)           (None, 64)            0           dense_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "output (Dense)                   (None, 1)             65          drop_dense_2[0][0]               \n",
      "====================================================================================================\n",
      "Total params: 681,601\n",
      "Trainable params: 681,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Configure model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "modelcheckpoint = ModelCheckpoint(filepath=output_dir+\"/weights.{epoch:02d}.hdf5\")\n",
    "if not os.path.exists(output_dir):\n",
    "    os.makedirs(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/4\n",
      "25000/25000 [==============================] - 192s - loss: 0.5019 - acc: 0.7187 - val_loss: 0.2899 - val_acc: 0.8794\n",
      "Epoch 2/4\n",
      "25000/25000 [==============================] - 7s - loss: 0.2508 - acc: 0.9014 - val_loss: 0.2555 - val_acc: 0.8952\n",
      "Epoch 3/4\n",
      "25000/25000 [==============================] - 7s - loss: 0.1719 - acc: 0.9369 - val_loss: 0.3117 - val_acc: 0.8715\n",
      "Epoch 4/4\n",
      "25000/25000 [==============================] - 7s - loss: 0.1291 - acc: 0.9536 - val_loss: 0.3019 - val_acc: 0.8892\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa297a089e8>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# start with conv_1 only and no concat: 89.1% validation accuracy in epoch 2, as earlier notebook\n",
    "# add conv_2: 89.5% in epoch 3\n",
    "# add conv_3: ditto\n",
    "# add dense_2: ditto in epoch 2\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(output_dir+\"/weights.01.hdf5\") # zero-indexed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_hat = model.predict(x_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEOJJREFUeJzt3X2snnV9x/H3Ryo+I8UW41pcMVYnmiyyButMnLMGAQ3l\nD1lq5qikWRPHnHNmE7c/uviQ6J7YSNStk85inMiYGY3iSIcQt0XQIg55GGkHDM5gUi1WN+JD9bs/\n7l/dkd9pe3Ouc87d075fycm5ru/1u+77++Oc9tPr4b5IVSFJ0nRPmnQDkqSjj+EgSeoYDpKkjuEg\nSeoYDpKkjuEgSeoYDpKkzhHDIcm2JI8kuWNa7ZQkO5Psbt+XtnqSXJ5kT5Lbk5w5bZ+NbfzuJBun\n1X8hydfbPpcnyVxPUpL0xIxz5PBx4JzH1S4Fbqiq1cANbR3gXGB1+9oMfBRGYQJsAV4BnAVsORgo\nbczmafs9/r0kSQtsyZEGVNUXk6x6XHk98Jq2vB24CXh3q19Zo49d35zk5CTPa2N3VtU+gCQ7gXOS\n3AScVFVfavUrgQuAzx+pr2XLltWqVY9vS5qw79wz+n7SiyfbhzSDW2+99ZtVtXycsUcMh0N4blU9\nDFBVDyc5tdVXAA9OGzfVaoerT81Qn1GSzYyOMnj+85/Prl27Ztm+NE/+6TWj76+7aZJdSDNK8p/j\njp3rC9IzXS+oWdRnVFVbq2pNVa1Zvnys8JMkzcJsw+Eb7XQR7fsjrT4FnDZt3ErgoSPUV85QlyRN\n0GzDYQdw8I6jjcC10+oXtbuW1gL72+mn64GzkyxtF6LPBq5v276bZG27S+miaa8lSZqQI15zSPIp\nRheUlyWZYnTX0QeBq5NsAh4ALmzDrwPOA/YAjwEXA1TVviTvA77Sxr334MVp4G2M7oh6GqML0Ue8\nGC1Jml/j3K305kNsWjfD2AIuOcTrbAO2zVDfBbzsSH1IkhaOn5CWJHUMB0lSx3CQJHUMB0lSZ7af\nkF7UVl36uYm87/0ffMNE3leSniiPHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJHcNBktQxHCRJHcNBktQxHCRJnePyqaySNNSx/nRnjxwkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSZ1B4ZDknUnuTHJHkk8l\neWqS05PckmR3kk8nObGNfUpb39O2r5r2Ou9p9XuSvH7YlCRJQ806HJKsAH4LWFNVLwNOADYAHwIu\nq6rVwKPAprbLJuDRqnohcFkbR5Iz2n4vBc4BPpLkhNn2JUkabuhppSXA05IsAZ4OPAy8Frimbd8O\nXNCW17d12vZ1SdLqV1XV96vqPmAPcNbAviRJA8w6HKrqv4A/AR5gFAr7gVuBb1fVgTZsCljRllcA\nD7Z9D7Txz5len2EfSdIEDDmttJTRv/pPB34GeAZw7gxD6+Auh9h2qPpM77k5ya4ku/bu3fvEm5Yk\njWXIaaXXAfdV1d6q+iHwGeAXgZPbaSaAlcBDbXkKOA2gbX82sG96fYZ9fkpVba2qNVW1Zvny5QNa\nlyQdzpBweABYm+Tp7drBOuAu4EbgTW3MRuDatryjrdO2f6GqqtU3tLuZTgdWA18e0JckaaAlRx4y\ns6q6Jck1wFeBA8BtwFbgc8BVSd7fale0Xa4APpFkD6Mjhg3tde5McjWjYDkAXFJVP5ptX5Kk4WYd\nDgBVtQXY8rjyvcxwt1FVfQ+48BCv8wHgA0N6kSTNHT8hLUnqGA6SpI7hIEnqGA6SpI7hIEnqGA6S\npI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7h\nIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnq\nGA6SpI7hIEnqGA6SpM6gcEhycpJrkvx7kruTvDLJKUl2Jtndvi9tY5Pk8iR7ktye5Mxpr7Oxjd+d\nZOPQSUmShhl65PAXwD9W1c8BPw/cDVwK3FBVq4Eb2jrAucDq9rUZ+ChAklOALcArgLOALQcDRZI0\nGbMOhyQnAa8GrgCoqh9U1beB9cD2Nmw7cEFbXg9cWSM3AycneR7wemBnVe2rqkeBncA5s+1LkjTc\nkCOHFwB7gb9JcluSjyV5BvDcqnoYoH0/tY1fATw4bf+pVjtUXZI0IUPCYQlwJvDRqno58L/8/ymk\nmWSGWh2m3r9AsjnJriS79u7d+0T7lSSNaUg4TAFTVXVLW7+GUVh8o50uon1/ZNr406btvxJ46DD1\nTlVtrao1VbVm+fLlA1qXJB3OrMOhqv4beDDJi1tpHXAXsAM4eMfRRuDatrwDuKjdtbQW2N9OO10P\nnJ1kabsQfXarSZImZMnA/d8OfDLJicC9wMWMAufqJJuAB4AL29jrgPOAPcBjbSxVtS/J+4CvtHHv\nrap9A/uSJA0wKByq6mvAmhk2rZthbAGXHOJ1tgHbhvQiSZo7fkJaktQxHCRJHcNBktQxHCRJHcNB\nktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQx\nHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJ\nHcNBktQxHCRJHcNBktQxHCRJHcNBktQZHA5JTkhyW5LPtvXTk9ySZHeSTyc5sdWf0tb3tO2rpr3G\ne1r9niSvH9qTJGmYuThyeAdw97T1DwGXVdVq4FFgU6tvAh6tqhcCl7VxJDkD2AC8FDgH+EiSE+ag\nL0nSLA0KhyQrgTcAH2vrAV4LXNOGbAcuaMvr2zpt+7o2fj1wVVV9v6ruA/YAZw3pS5I0zNAjhz8H\nfg/4cVt/DvDtqjrQ1qeAFW15BfAgQNu+v43/SX2GfX5Kks1JdiXZtXfv3oGtS5IOZdbhkOSNwCNV\ndev08gxD6wjbDrfPTxertlbVmqpas3z58ifUryRpfEsG7Psq4Pwk5wFPBU5idCRxcpIl7ehgJfBQ\nGz8FnAZMJVkCPBvYN61+0PR9JEkTMOsjh6p6T1WtrKpVjC4of6GqfhW4EXhTG7YRuLYt72jrtO1f\nqKpq9Q3tbqbTgdXAl2fblyRpuCFHDofybuCqJO8HbgOuaPUrgE8k2cPoiGEDQFXdmeRq4C7gAHBJ\nVf1oHvqSJI1pTsKhqm4CbmrL9zLD3UZV9T3gwkPs/wHgA3PRiyRpOD8hLUnqGA6SpI7hIEnqGA6S\npI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7h\nIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnq\nGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM6swyHJaUluTHJ3kjuTvKPVT0myM8nu9n1pqyfJ5Un2JLk9\nyZnTXmtjG787ycbh05IkDTHkyOEA8K6qegmwFrgkyRnApcANVbUauKGtA5wLrG5fm4GPwihMgC3A\nK4CzgC0HA0WSNBmzDoeqeriqvtqWvwvcDawA1gPb27DtwAVteT1wZY3cDJyc5HnA64GdVbWvqh4F\ndgLnzLYvSdJwc3LNIckq4OXALcBzq+phGAUIcGobtgJ4cNpuU612qLokaUIGh0OSZwJ/D/x2VX3n\ncENnqNVh6jO91+Yku5Ls2rt37xNvVpI0lkHhkOTJjILhk1X1mVb+RjtdRPv+SKtPAadN230l8NBh\n6p2q2lpVa6pqzfLly4e0Lkk6jCF3KwW4Ari7qv5s2qYdwME7jjYC106rX9TuWloL7G+nna4Hzk6y\ntF2IPrvVJEkTsmTAvq8Cfg34epKvtdrvAx8Erk6yCXgAuLBtuw44D9gDPAZcDFBV+5K8D/hKG/fe\nqto3oC9J0kCzDoeq+hdmvl4AsG6G8QVccojX2gZsm20vkqS55SekJUkdw0GS1DEcJEkdw0GS1DEc\nJEkdw0GS1DEcJEkdw0GS1BnyCWlJmqhVl35u0i0cswyHBTTJX+T7P/iGib23pMXH00qSpI7hIEnq\nGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7hIEnqGA6SpI7PVpI0mA/AO/YYDseJSf3h\n9YF/0uLkaSVJUsdwkCR1PK2keeX/w2LheN5fc8lw0DFrEn9ZXvWCbwGwwb+otch5WkmS1DEcJEkd\nw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DlqwiHJOUnuSbInyaWT7keSjmdHRTgkOQH4MHAucAbw5iRn\nTLYrSTp+HRXhAJwF7Kmqe6vqB8BVwPoJ9yRJx62jJRxWAA9OW59qNUnSBBwtz1bKDLXqBiWbgc1t\n9X+S3DPL91sGfHOW+y5WznkBvPInS29cyLc9yJ/xcSAfGjTnnx134NESDlPAadPWVwIPPX5QVW0F\ntg59syS7qmrN0NdZTJzzse94my845/l0tJxW+gqwOsnpSU4ENgA7JtyTJB23joojh6o6kOQ3geuB\nE4BtVXXnhNuSpOPWUREOAFV1HXDdAr3d4FNTi5BzPvYdb/MF5zxvUtVd95UkHeeOlmsOkqSjyDEb\nDkd6HEeSpyT5dNt+S5JVC9/l3Bpjzr+T5K4ktye5IcnYt7UdrcZ97EqSNyWpJIv+zpZx5pzkV9rP\n+s4kf7vQPc61MX63n5/kxiS3td/v8ybR51xJsi3JI0nuOMT2JLm8/fe4PcmZc95EVR1zX4wuav8H\n8ALgRODfgDMeN+Y3gL9syxuAT0+67wWY8y8DT2/Lbzse5tzGPQv4InAzsGbSfS/Az3k1cBuwtK2f\nOum+F2DOW4G3teUzgPsn3ffAOb8aOBO44xDbzwM+z+gzYmuBW+a6h2P1yGGcx3GsB7a35WuAdUlm\n+jDeYnHEOVfVjVX1WFu9mdHnSRazcR+78j7gj4DvLWRz82ScOf868OGqehSgqh5Z4B7n2jhzLuCk\ntvxsZvic1GJSVV8E9h1myHrgyhq5GTg5yfPmsodjNRzGeRzHT8ZU1QFgP/CcBelufjzRR5BsYvQv\nj8XsiHNO8nLgtKr67EI2No/G+Tm/CHhRkn9NcnOScxasu/kxzpz/EHhLkilGdz2+fWFam5h5f+TQ\nUXMr6xwb53EcYz2yYxEZez5J3gKsAX5pXjuaf4edc5InAZcBb12ohhbAOD/nJYxOLb2G0dHhPyd5\nWVV9e557my/jzPnNwMer6k+TvBL4RJvzj+e/vYmY97+/jtUjh3Eex/GTMUmWMDoUPdxh3NFurEeQ\nJHkd8AfA+VX1/QXqbb4cac7PAl4G3JTkfkbnZncs8ovS4/5uX1tVP6yq+4B7GIXFYjXOnDcBVwNU\n1ZeApzJ67tKxaqw/70Mcq+EwzuM4dgAb2/KbgC9Uu9KzSB1xzu0Uy18xCobFfh4ajjDnqtpfVcuq\nalVVrWJ0neX8qto1mXbnxDi/2//A6OYDkixjdJrp3gXtcm6NM+cHgHUASV7CKBz2LmiXC2sHcFG7\na2ktsL+qHp7LNzgmTyvVIR7HkeS9wK6q2gFcwejQcw+jI4YNk+t4uDHn/MfAM4G/a9feH6iq8yfW\n9EBjzvmYMuacrwfOTnIX8CPgd6vqW5Prepgx5/wu4K+TvJPR6ZW3LuZ/7CX5FKPTgsvadZQtwJMB\nquovGV1XOQ/YAzwGXDznPSzi/36SpHlyrJ5WkiQNYDhIkjqGgySpYzhIkjqGgySpYzhIkjqGgySp\nYzhIkjr/B4ssvuVhCgI3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa293ef2a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(y_hat)\n",
    "_ = plt.axvline(x=0.5, color='orange')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'96.16'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"{:0.2f}\".format(roc_auc_score(y_valid, y_hat)*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
